import os
from glob import glob

from tqdm import tqdm
import hashlib

from config import config


def gen_hash(text):
    # 创建一个SHA256哈希对象
    hash_object = hashlib.sha256()

    # 更新哈希对象
    hash_object.update(text.encode())

    # 获取哈希值（十六进制字符串）
    hex_dig = hash_object.hexdigest()
    return hex_dig



# 控制程序
def main_data():
    path_list = glob(config["train_data_set_path"])
    new_path=dict()
    for one in tqdm(path_list):
        with open(one, "r", encoding="utf-8") as f:
            one_data = f.read()
            new_path[gen_hash(one_data)]=one
    new_path={k:v for v,k in new_path.items()}
    for one in tqdm(path_list):
        if new_path.get(one,None) is None:
            os.remove(one)



if __name__ == '__main__':
    main_data()
